#########################################################################
#									#
# Name:	 	fsmon.pm (Perl Module)					#
#									#
# Completed:	8th October, 2003.					#
#									#
# Updated:	4th January, 2009.					#
#									#
# Author:	Simon Edwards, Proprius Consulting Ltd.			#
#									#
# Arguments:	N/A							#
#									#
# Purpose:	This modules is a LEMS module that is used by the	#
#		cluster to check the functionality of the storage for	#
#		a package and then to take actions to being 		#
#		automated resynchronisation when possible.		#
#									#
# Limitations:	1 - Lots probably.					#
#									#
# Environment:	None.							#
#									#
# Exit Codes:	N/A							#
# 									#
#########################################################################

#################################################################
# The fsmonitor is THE MOST CRITICAL MONITOR.			#
# It is responsible for automatically restoring data 		#
# consistency for file systems belonging to a package.		#
#################################################################

package fsmon;

$fsmon::serverfails_max=3;

BEGIN {
	push @INC,"/usr/local/cluster/lib/perl";
}

use Net::Ping;
use XML::Simple;
use mdstat;
use fsmap;
use cld_client;
use Sys::Hostname;
use Data::Dumper;
use cldlockc;
use clutils qw(validate_cluster_cfg get_allocated_resource);
use clnet qw(CLNET_return_static_ips CLNET_return_network_static_ip);

#################################################################
# We drag in details of the application configuration, 		#
# but not the status information at start up...			#
#################################################################
sub mapdevicename {
my ($x,$y);
my $devnum=$_[0];

	if($devnum<26) {
		$x=chr(ord('a')+$devnum);
		return $x;
	}
	$x=chr(ord('a')+int($devnum/26));
	$y=chr(ord('a')+$devnum%26);
	return "$x$y";
}

sub new {
my $self={};
my $application=$_[1];
my ($err,@errs,$ip,$fdd,$cline,@f,@nodes,$xml2,$xml,$conn);

	$self->{APPLICATION_NAME}=$application;
	$self->{LNODE}=hostname;

	$self->{RAIDCFG}="/cluster/control/.resources/mddata/$application";

	#################################################################
	# Load in the cluster configuration information.		#
	#################################################################

	if(exists($ENV{ARG_CONFIG})) {
		$self->{ARG_CONFIG}=$ENV{ARG_CONFIG};
	} else {
		$self->{ARG_CONFIG}="/etc/cluster/clconf.xml";
	}

	($xml,$err,@errs)=validate_cluster_cfg($self->{ARG_CONFIG});
	if(!defined($xml)) {
		for (@errs) {
			::errmsg($_);
		}
		exit($err);
	}
	::logmsg("Validated cluster configuration.");
	my $_p=get_allocated_resource(hostname,"PORT","__CLUSTER__","%%CLLOCKD",undef);
	if($_p>0) {
		$self->{LOCK_PORT}=$_p;
		::logmsg("Cluster may use locking ... will check on first run.");
	} else {
		::logwarn("No locking port defined in cluster configuration.");
		$self->{LOCK_PORT}=0;
	}
	$self->{LOCK_CONN}=undef;

	$self->{DATADETAIL}=$xml->{global}->{datadetail};
	::logmsg("Data replication protocol in use: ".$self->{DATADETAIL});

	(@nodes)=keys(%{$xml->{node}});
	$xml2=XMLin("/etc/cluster/$application/appconf.xml");
	if(! defined $xml2) {
		#########################################################
		# The error message is available via the parent.	#
		#########################################################
		errmsg("Syntax error in appconf.xml file for application $application",50);
	}
	if(hostname ne $nodes[0]) {
		$self->{RNODE}=$nodes[0];
	} else {
		$self->{RNODE}=$nodes[1];
	}

	foreach (keys(%{$xml->{node}})) {
		if($_ eq hostname) {
			$self->{LOCALIP}=CLNET_return_network_static_ip(node=>hostname,network=>$xml->{global}->{drbd_network});
		} else {
			$self->{REMOTEIP}=CLNET_return_network_static_ip(node=>$self->{RNODE},network=>$xml->{global}->{drbd_network});
		}
	}

	#################################################################
	# Try to get a connection to the cldaemon - we need this to	#
	# send messages to attempt synchronisation of resources...	#
	#################################################################
	$self->{DAEMON}=undef;
	$self->{RDAEMON}=undef;
	$self->{STATE}=10;
	$self->{SUBSTATE}=0;
	$self->{LASTLOCALCHECK}=0;
	$self->{INSTANCECOUNT}=0;

	#################################################################
	# We might need information from the config throughout the	#
	# course of running, so keep its configuration around.		#
	#################################################################
	$self->{PACKAGE_CONF}=$xml2;
	$self->{CLUSTER_CONF}=$xml;

	#################################################################
	# Objects to scan the raid devices and fsmap for this		#
	# application.							#
	#################################################################
	$self->{FSMAP}=new fsmap(FILE => => "/cluster/control/.resources/fsmap/$application",
				 APPLICATION => $application);
	bless $self;
	return $self;
}

#########################################################################
# When the check is called the object has at least the following	#
# fields:								#
# APPLICATION_NAME	The actual name of the application.		#
# PACKAGE_CONF		A reference to a hash describing the package	#
#			configuration XML.				#
# STATE			An indication of what status we had when we	#
# 			were last called:				#
# 			0 - I'm new here - nothing done as yet.		#
# 			1 - Currently syncing MD defined in MDSYNC	#
# 			2 - Nothing to do last time...			#
#									#
#########################################################################

sub _getlock {
my $self=shift;
my ($lock,$lockr,$cldlock_client);
my ($type,$time);

	$cldlock_client=$self->{LOCK_CONN};
	return if !defined($cldlock_client);

	$type=$_[0];
	$time=$_[1] if exists($_[1]);
	$time=10 if !defined($time);

	($lock,$lockr)=$cldlock_client->acquire(
		PID	=> $PID, 
		CPROC	=> "fsmon-".$self->{APPLICATION_NAME},
		HOLD	=> $time,
		LOCK	=> $type);
	if(!$lock) {
		::logwarn("Unable to acquire lock: $lockr");
	}
}

sub _releaselock {
my $self=shift;
my ($lock,$lockr,$cldlock_client);
my ($type);

	$cldlock_client=$self->{LOCK_CONN};
	return if !defined($cldlock_client);
	$type=$_[0];
	($lock,$lockr)=$cldlock_client->release(
		PID	=> $PID, 
		CPROC	=> "fsmon-".$self->{APPLICATION_NAME},
		LOCK	=> $type);
	if(!$lock) {
		::logwarn("Unable to release lock: $lockr");
	}
}

#########################################################################
# This routine is used when the data replication is DRBD. In this	#
# case things are much more straightforward. Essentially managed	#
# by a series of flags...						#
# [1] have_remote_connection 0|1	Whether a remote connection	#
#					to the IP address for DRBD	#
#					traffic is available on the	#
#					remote machine.			#
# [2] have_local_daemon 0|1	Whether a local daemon connection is	#
#				available.				#
# [3] have_remote_daemon 0|1	Whether a remote daemon conncetion is	#
#				available.				#
# [4] have_remote_devs 0|1	Whether the DRBD devices are defined on	#
#				remote host.				#
# [5] have_sync_data 0|1	Whether all data is synchronised or	#
#				not.					#
#									#
# Algorithm in use:							#
#									#
# STATE == 10								#
# Create all flags as 0, and STATE to 0 - then return 0.		#
#									#
# STATE == 0								#
#	<short sleep state - 5 seconds between checks>			#
#   If have_local_daemon == 0						#
#	Attempt local connection and set have_local_daemon to 1 if so.	#
#	If not available set STATE=1					#
#	Return 0							#
#   If have_remote_daemon == 0						#
#	Attempt remote connection and set have_remote_daemon to 1 if so.#
#	If not available set STATE=1					#
#	Return 0							#
#   If have_remote_devs == 0						#
#	Attempt remote daemon start (via START_NBD message).		#
#	If ok then set have_remote_dev to 1.				#
#	If not ok then set STATE=1					#
#   If have_sync_data == 0						#
#	Check status of synchronisation.				#
#	If complete set STATE=1						#
#	If sync'ing set STATE=1						#
#	If all unsync, set all vars to 0, to recheck everything.	#
#									#
# STATE == 1								#
#	<long sleep state - 60 seconds between checks>			#
#	set have_sync_data=0						#
#	set STATE to 0.							#
#									#
#									#
# The return values used are:						#
# 0 - ok short sleep							#
# 1 - ok long sleep							#
#									#
#########################################################################


sub check {
my $self=shift;
my ($resp,$xml,$xml2,$conn,$cip,$p,$ctype);
my ($lnode,$rnode);

	$rnode=$self->{RNODE};
	$lnode=$self->{LNODE};

	$xml=$self->{CLUSTER_CONF};
	$xml2=$self->{PACKAGE_CONF};

	#################################################################
	# When in state 10 we simply set up everything.			#
	#################################################################
	if($self->{STATE}==10) {
		$self->{have_remote_connection}=0;
		$self->{have_local_daemon}=0;
		$self->{have_remote_daemon}=0;
		$self->{have_remote_devs}=0;
		$self->{have_sync_data}=0;
		$self->{STATE}=0;
		return 0;
	}
	if($self->{STATE}==1) {
		$self->{have_sync_data}=0;
		$self->{STATE}=0;
		return 1;
	}

	#################################################################
	# The rest of the code here is do deal with state == 0, where	#
	# all the major checks are performed.				#
	#################################################################
	if($self->{have_remote_connection}==0) {
		$cip=$self->{REMOTEIP};
		::logmsg("[DRBD] Attempting to communicate to IP $cip(icmp)");
		$p=new Net::Ping("icmp",2);
		if(!($p->ping($cip))) {
			::logmsg("[DRBD] Can not communicate to DRBD IP address.");
			return 1;
		}
		::logmsg("[DRBD] Connected remote server via DRBD IP.");
		$self->{have_remote_connection}=1;
		return 0;
	}

	if($self->{have_local_daemon}==0) {

		::logmsg("[DRBD] Attempting to contact local cldaemon...");
		my $_p=get_allocated_resource(hostname,"PORT","__CLUSTER__","%%CLDAEMON",undef);
		$conn=new cld_client(PORT => $_p, INET => "127.0.0.1", KEY => $xml->{global}->{key});
		if(defined($conn)) {
			my $resp;

			$resp=$conn->send_msg(MSG => "ECHO check=y", TIMEOUT => 10);
			$conn->close_session;
			if(defined($resp)) {
				$self->{DAEMON}=$conn;
				::logmsg("[DRBD] Local cldaemon now started!");
				$self->{have_local_daemon}=1;
				return 0;
			} else {
				#################################
				# Daemon not running, try again	#
				# in 60 seconds.		#
				#################################
				$self->{DAEMON}=undef;
				return 1;
			}
		} else {
			$self->{DAEMON}=undef;
			return 1;
		}
	}
	#################################################################
	# We now check to see if a remote daemon connection is		#
	# available, and if not, attempt to get it.			#
	#################################################################
	if($self->{have_remote_daemon}==0) {
		$self->{INSTANCECOUNT}=0;
		::logmsg("[DRBD] Attempting to contact remote cldaemon...");
		@iflist=CLNET_return_static_ips(node=>$rnode);
		$ctype="icmp";
		$p=new Net::Ping($ctype,2);

		my $_p=get_allocated_resource(hostname,"PORT","__CLUSTER__","%%CLDAEMON",undef);
		for $cip (@iflist) {
			if(!($p->ping($cip))) {
				::logmsg("[DRBD] IP address $cip($ctype) not suitable for $rnode");
				next;
			}
			$conn=new cld_client(PORT => $_p, INET => $cip, KEY => $xml->{global}->{key});
			if(defined($conn)) {
				my $resp;

				$resp=$conn->send_msg(MSG => "ECHO check=y", TIMEOUT => 10);
				$conn->close_session;
				if(defined($resp)) {
					::logmsg("[DRBD] Contacted cldaemon via $cip for $rnode");
					$self->{RDAEMON}=$conn;
					$self->{have_remote_daemon}=1;
					return 0;
				} else {
					::logmsg("[DRBD] Daemon did not send response from $rnode!");
				}
			} else {
				::logmsg("[DRBD] Unable to contact daemon on $cip:".$xml->{global}->{PORT});
			}
		}
		#################################################################
		# If we get here no connection is available, so			#
		# do nothing, but wait for 60 seconds before doing anything	#
		# else.								#
		#################################################################
		$self->{RDAEMON}=undef;
		return 1;
	}

	#################################################################
	# Check local/remote devs are working ok...			#
	#################################################################

	if($self->{have_remote_devs}==0) {
		::logmsg("[DRBD] Attempting to check DRBD devices locally...");

		$resp=$self->{DAEMON}->send_msg(MSG => "CHECK_NBD type=client app=".$self->{APPLICATION_NAME}, TIMEOUT => 10);
		$self->{DAEMON}->close_session;
		if(defined($resp)) {
			::logmsg("[DRBD] Response from CHECK_NBD on localhost was $resp");
			if($resp ne "OK") {
				$resp=$self->{DAEMON}->send_msg(MSG => "START_NBD renewonstandalone=yes app=".$self->{APPLICATION_NAME}, TIMEOUT => 10);
				$self->{DAEMON}->close_session;
				if(!defined($resp)) {
					::logwarn("[DRBD] START_NBD returned no results.");
					$self->{STATE}=10;
					$self->{DAEMON}=undef;
					$self->{RDAEMON}=undef;
					return 0;
				} else {
					if($resp eq "OK") {
						::logmsg("[DRBD] Local server started DRBD services.");
						# return 0;
					} else {
						::logwarn("[DRBD] Local server returned $resp.");
						$self->{STATE}=10;
						$self->{DAEMON}=undef;
						$self->{RDAEMON}=undef;
						return 0;
					}
				}
			} else {
				#########################################
				# Check / start remote devices...	#
				# 0196: Include the instance count,	#
				#	starting at 0 - this is passed	#
				#	to the cldaemon so when it	#
				#	reaches 3 or more a complete	#
				#	data invalidate is performed.	#
				#########################################
				::logmsg("[DRBD] Attempting to check DRBD devices remotely...");

				$resp=$self->{RDAEMON}->send_msg(MSG => "CHECK_NBD type=client app=".$self->{APPLICATION_NAME}, TIMEOUT => 10);
				$self->{RDAEMON}->close_session;
				if(defined($resp)) {
					::logmsg("[DRBD] Response from CHECK_NBD on remote host was $resp");
					if($resp ne "OK") {
						my $instance=$self->{INSTANCECOUNT};
						$resp=$self->{RDAEMON}->send_msg(MSG => "START_NBD instancecount=$instance app=".$self->{APPLICATION_NAME}, TIMEOUT => 10);
						$self->{INSTANCECOUNT}++;
						$self->{RDAEMON}->close_session;
						if(!defined($resp)) {
							::logwarn("[DRBD] START_NBD returned no results.");
							$self->{STATE}=10;
							$self->{DAEMON}=undef;
							$self->{RDAEMON}=undef;
							return 0;
						} else {
							if($resp eq "OK") {
								::logmsg("[DRBD] Remote server started DRBD services.");
								return 0;
							} else {
								::logwarn("[DRBD] Remote server returned $resp.");
								$self->{STATE}=10;
								$self->{DAEMON}=undef;
								$self->{RDAEMON}=undef;
								return 0;
							}
						}
					}
				} else {
					::logwarn("[DRBD] CHECK_NBD from remote server returned no results.");
					$self->{STATE}=10;
					$self->{DAEMON}=undef;
					$self->{RDAEMON}=undef;
					return 0;
				}
			}
		} else {
			::logwarn("[DRBD] CHECK_NBD from local host returned no results.");
			$self->{STATE}=10;
			$self->{DAEMON}=undef;
			$self->{RDAEMON}=undef;
			return 0;
		}
	}
	::logmsg("[DRBD] Nothing to do...");
	$self->{STATE}=10;
	$self->{DAEMON}=undef;
	$self->{RDAEMON}=undef;
	return 1;
}

sub stat {
my $self=shift;
my ($mdlist,$mdinfo,$str,$cdev,$ref);
my $xml;
my ($fsinfo,$l,$c,$r,@s,@r2);

	$xml=$self->{CLUSTER_CONF};
	$str=sprintf("%d,%d\n",$self->{STATE},$self->{SUBSTATE});

	$fsinfo=new fsmap(FILE	=> "/etc/cluster/.resources/fsmap/" . $self->{APPLICATION_NAME},
		APPLICATION => $self->{APPLICATION_NAME}, DATADETAIL => "drbd");

	$c=$fsinfo->count;
	$l=0;
	while($l<$c) {
		$r=$fsinfo->get_sync_state(RECNO => $l);
		if(defined($r)) {
			@r2=$fsinfo->get(RECNO => $l);
			@s=split(/:/,$r);
			if($s[6] eq "syncing") {
				$str.="$r2[2],$s[6],drbd,$s[4],$s[7],$s[8],$s[9]\n";
			} else {
				$str.="$r2[2],$s[6],drbd,$s[4]\n";
			}
		}
		$l++;
	}
	return $str;
}
1;

